import requests
from requests.exceptions import RequestException
from lxml import etree
from bs4 import BeautifulSoup as bs
from pyquery import PyQuery as pq
import time, json

# 抓取页面信息
def getPage(url):
    try:
        headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36',
        }
        res = requests.get(url, headers=headers)
        if res.status_code == 200:
            return res.text
        else:
            return None
    except RequestException:
        return None

# 解析网页信息
'''
def parsePage(content):
    # print(content)
    html = etree.HTML(content)
    items = html.xpath('//div[@class="item"]')
    for item in items:
        yield {
            'index':item.xpath('.//div/em[@class=""]/text()')[0],
            'pic':item.xpath('.//img[@width="100"]/@src')[0],
            'title':item.xpath('.//span[@class="title"]/text()')[0],
            'actor':item.xpath('.//p[@class=""]/text()')[0].strip().replace('\xa0\xa0\xa0',';'),
            'score':item.xpath('.//span[@class="rating_num"]/text()')[0],
            'year':item.xpath('.//p[@class=""]/text()')[1].strip()[:4],
        }
'''

'''
def parsePage(content):
    # print(content)
    html = bs(content, "lxml")
    items = html.find_all(name='div',attrs={'class':'item'})
    for item in items:
        yield {
            'index':item.em.string,
            'pic':item.find(name='img',attrs={'width':'100'}).attrs['src'],
            'title':item.find(name='span',attrs={'class':'title'}).string,
            'actor':item.select('div.bd p')[0].get_text(),
            'score':item.select('div.star span')[1].string,
        }
'''

def parsePage(content):
    html = pq(content)
    items = html('div.item')
    for item in items.items():
        yield {
            # 'index':item.find('div.pic em').text(),
            # 'pic':item.find('div.pic img').attr('src'),
            # 'title':item.find('div.hd span.title').text(),
            'actor':item.find('div.bd p:eq(0)').text(),
            # 'score':item.find('div.star span.rating_num').text(),
            # 'year':item.find('div.bd p:eq(1)').text().strip().replace('\xa0\xa0\xa0',';')[:4],
        }

# 解析内容打印或写入文件
def writeFile(content):
    with open('./result.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False)+'\n')

# 主函数
def main(offset):
    url = 'https://movie.douban.com/top250?start=%d' % (offset)
    html = getPage(url)
    if html:
        for item in parsePage(html):
            # writeFile(item)
            print(item)

if __name__ == '__main__':
    for offset in range(1):
        main(offset=offset*25)
        time.sleep(1)